from math import sqrt
import jieba

def similarity(seg1, seg2):
    all_list = []
    all_list.append(seg1)
    all_list.append(seg2)
    corpus = set(all_list)
    corpus_dict = dict(zip(corpus, range(len(corpus))))
    vec1 = vector_rep(seg1, corpus_dict)
    vec2 = vector_rep(seg2, corpus_dict)
    cosine_sim = similarity_with_2_sents(vec1, vec2)
    return cosine_sim

# 建立句子的向量表示
def vector_rep(text, corpus_dict):
    vec = []
    for key in corpus_dict.keys():
        if key in text:
            vec.append((corpus_dict[key], text.count(key)))
        else:
            vec.append((corpus_dict[key], 0))
    vec = sorted(vec, key= lambda x: x[0])
    return vec


def similarity_with_2_sents(vec1, vec2):
    inner_product = 0
    square_length_vec1 = 0
    square_length_vec2 = 0
    for tup1, tup2 in zip(vec1, vec2):
        inner_product += tup1[1]*tup2[1]
        square_length_vec1 += tup1[1]**2
        square_length_vec2 += tup2[1]**2

    return (inner_product/sqrt(square_length_vec1*square_length_vec2))

f = open('data.txt', 'r', encoding='utf-8')
#seg_file = open('seg_result.txt', 'w')
seg_list = []
while True:
    content = f.readline()
    if content == '':
        break
    seg = jieba.lcut(content)
    #print(seg)
    #seg_file.write(str(seg) + '\n')
    seg_list.append(seg)
f.close()
#seg_file.close()
print('分词成功')

for i in range(len(seg_list)):
    str1 = str(seg_list[i][0])
    if str1 != '000':
        for j in range(len(seg_list)):
            if i < j:
                print('开始第 %s 轮数据处理，%s' % (i, j))
                str2 = str(seg_list[j][0])
                if str2 != '000':
                    cosine_sim = similarity(str1, str2)
                    if cosine_sim > 0.7:
                        seg_list[j][0] = '000'
print('句子相似性比较处理完成')
result_list = set(seg_list)
while '000' in seg_list:
    seg_list.remove('000')
result_file = open('result.txt', 'w')
for l in seg_list:
    result_file.writelines(l)
result_file.close()
print('结果写入完成')

